#!/usr/bin/env python
# coding: utf-8
import pandas as pd
import re
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
def xl_dict(xl):
    """
    筛选excel数据生成字典
    """
    ddf = dict()
    for sheet_name in xl.sheet_names:
        if sheet_name in ['dataset', 'dictionary']:
            ddf[sheet_name] = xl.parse(sheet_name, index_col=0)
        else:
            ddf[sheet_name] = xl.parse(sheet_name)
    return ddf

#  初步生成word_freq档
def run_word_freq(xl_df):
    """
    生成word_freq档
    """
    df = xl_df
    corpus = df.BOW_JB.to_list()
    words_in_a_cropus = []
    for doc in corpus:
        words_in_a_cropus.extend(doc)  
    df_voc = pd.DataFrame({'words': words_in_a_cropus}, index = list(range(len(words_in_a_cropus))))
    df_v = df_voc.groupby(by="words").size().reset_index(name='counts').sort_values(by='counts', ascending=False)
    df_v['rank'] = df_v['counts'].rank(method='dense', ascending=False)
    df_v['类别'] = ""
    df_v['修正'] = ""
    df_v['memo'] = ""
    return df_v



